from pymongo import MongoClient
from bs4 import BeautifulSoup
from pprint import pprint
import requests
import datetime
import json
import time
import re

server_client = MongoClient('127.0.0.1', 27017)

server_db = server_client['knx_posts_db']
offical_posts_coll = server_db['offical_posts_coll']

headers = {
    'authority': 'www.basf.com',
    'method': 'GET',
    'path': '/cn/zh/company/career/jobs.html',
    'scheme': 'https',
    'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
    'accept-encoding': 'gzip, deflate, br',
    'cookie': 'SessionPersistence=CLIENTCONTEXT%3A%3DvisitorId%253Danonymous%7CPROFILEDATA%3A%3Davatar%253D%252Fetc%252Fdesigns%252Fdefault%252Fimages%252Fsocial%252Favatar.png%252Cpath%253D%252Fhome%252Fusers%252Fa%252Fanonymous%252Fprofile%252CisLoggedIn%253Dfalse%252CisLoggedIn_xss%253Dfalse%252CauthorizableId%253Danonymous%252CauthorizableId_xss%253Danonymous%252CformattedName%253Danonymous%252CformattedName_xss%253Danonymous; activeCampaignID=CN30000509; utag_main=v_id:016213a3175d0017b55e3c15e2af0407900850710093c$_sn:1$_ss:0$_st:1520750367973$ses_id:1520747878240%3Bexp-session$_pn:6%3Bexp-session; TS010e2b08=011bc2843b175fbb77fa14b3cf8874f8953325af67eb35aab4b3a9a5ec7816852bb1136c93e57e559175e7579c4c9cd0413475c989; WT_FPC=id=8cc52171-ce28-4b02-b5a3-e442e8cc9e9c:lv=1520723383081:ss=1520723383081; WT_FPC=id=8cc52171-ce28-4b02-b5a3-e442e8cc9e9c:lv=1520719785406:ss=1520723001461',
    'accept-language': 'en,zh-CN;q=0.9,zh;q=0.8,en-US;q=0.7',
    'cache-control': 'no-cache',
    'pragma': 'no-cache',
    'referer': 'https://www.basf.com/cn/zh/company/career.html',
    'upgrade-insecure-requests': '1',
    'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36'
}

json_headers = {
    'authority': 'www.basf.com',
    'method': 'GET',
    'path': '/data/wcms/rmk/jobs.list.zh_CN.json',
    'scheme': 'https',
    'accept': 'application/json, text/javascript, */*; q=0.01',
    'accept-encoding': 'gzip, deflate, br',
    'accept-language': 'en,zh-CN;q=0.9,zh;q=0.8,en-US;q=0.7',
    'cache-control': 'no-cache',
    'cookie': 'SessionPersistence=CLIENTCONTEXT%3A%3DvisitorId%253Danonymous%7CPROFILEDATA%3A%3Davatar%253D%252Fetc%252Fdesigns%252Fdefault%252Fimages%252Fsocial%252Favatar.png%252Cpath%253D%252Fhome%252Fusers%252Fa%252Fanonymous%252Fprofile%252CisLoggedIn%253Dfalse%252CisLoggedIn_xss%253Dfalse%252CauthorizableId%253Danonymous%252CauthorizableId_xss%253Danonymous%252CformattedName%253Danonymous%252CformattedName_xss%253Danonymous; activeCampaignID=CN30000509; WT_FPC=id=8cc52171-ce28-4b02-b5a3-e442e8cc9e9c:lv=1520719878903:ss=1520723001461; WT_FPC=id=8cc52171-ce28-4b02-b5a3-e442e8cc9e9c:lv=1520725717089:ss=1520725717089; utag_main=v_id:016213a3175d0017b55e3c15e2af0407900850710093c$_sn:2$_ss:1$_st:1520752945161$ses_id:1520751145161%3Bexp-session$_pn:1%3Bexp-session; TS010e2b08=011bc2843ba7e1c3e797f278ed19d82721b2adaab9939337f998808954624d4d71fc3fd63fcb87fa58daae85579f2f11efd4fb7e94',
    'pragma': 'no-cache',
    'referer': 'https://www.basf.com/cn/zh/company/career/jobs.html',
    'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36',
    'x-requested-with': 'XMLHttpRequest'
}


class POST():
    def __init__(self):
        self.scrapy()

    def scrapy(self):
        url = 'https://www.basf.com/data/wcms/rmk/jobs.list.zh_CN.json'
        r = requests.get(url, headers = json_headers)

        for jd in r.json()['jobs']:
            if jd['globalJobLevel'].find('应届毕业生') == -1 and jd['globalJobLevel'].find('学生') == -1 and jd['globalJobLevel'].find('在校生') == -1:
                continue

            if jd['location'].find('中国') == -1:
                continue

            name = jd['title']
            url = jd['url']
            location = jd['cityName']
            company = jd['company']
            date = jd['dateRaw']
            description = jd['description']

            item = {
                "url": url,
                'edu': '',
                'exp': [],
                'name': name,
                'date': date,
                'lang': '',
                'place': '',
                'major': '',
                'count': -1,
                'salary': [],
                'toSchool': True,
                'welfare': [],
                'funType': '',
                'company': company,
                'location': location,
                'industry': '化工',
                'keywords': [],
                'platform': 'offical',
                'searchKeyword': '',
                'description': description,
                'subIndustry': '',
                'stime': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
            }

            result = offical_posts_coll.replace_one({'company': company, 'name': name, 'location': location}, item, True)

            print(item['company'], item['name'])


POST()
